﻿
Imports System.Threading
Imports std
Imports storoomc.storclient
Imports rashare

Public Module racrawler
    Public Const defaultConfigfile As String = "racrawler.ini"

    Public servicethread As Thread = Nothing
    Public crawlthreads As vector(Of Thread) = Nothing
    Public mutex As Mutex = Nothing
    Public urls As vector(Of String) = Nothing
    Public rssreader As rssreader = Nothing

    Public Sub crawl()
        assert(Not urls Is Nothing, "did not read url list before crawling.")
        assert(Not mutex Is Nothing, "did not initalize mutex before crawling.")
        Dim talk As tcpTalk = Nothing
        Dim buff As bytes = Nothing
        Dim items As deque(Of String) = Nothing
        buff = New bytes()
        items = New deque(Of String)()

        While Not urls.empty
            assert(mutex.WaitOne(), "cannot lock mutex when get url.")
            If Not urls.empty() Then
                Dim url As String = Nothing
                copy(url, urls.back())
                urls.pop_back()
                mutex.ReleaseMutex()
                Dim content As String = Nothing
                If httpUtils.readHttpWeb(url, content, config.httpretrytimes) Then
                    If urlUtils.urlnormalize(url) Then
                        If rssreader.parse(content, items, url) Then
                            content = Nothing
                            Dim i As Int32
                            For i = 0 To items.size() - 1
                                content += items(i) + character.newline
                            Next
                            If Not isemptystring(content) Then
                                buff.copy(encoding.utf8.GetBytes(content))
                                If storclient.trymodify(talk, config.host, config.port _
                                                       , url, buff, config.storoomretrytimes) <> returnValue.success Then
                                    raiseError("cannot write content to storoom, url " + url + ", ignore." _
                                               , errorHandle.errorType.warning)
                                End If
                            End If
                        Else
                            raiseError("cannot parse rss content, ignore.", errorHandle.errorType.warning)
                        End If
                    Else
                        raiseError("cannot normalize url " + url + ", ignore.", errorHandle.errorType.warning)
                    End If
                Else
                    raiseError("cannot get http result from url " + url + ", ignore.", errorHandle.errorType.warning)
                End If
            Else
                mutex.ReleaseMutex()
                Exit While
            End If
        End While
    End Sub

    Public Sub startCrawlThreads(ByVal threadcount As Int32)
        assert(Not crawlthreads Is Nothing AndAlso crawlthreads.size() = 0 _
               , "should create crawlthreads and clear it before startCrawlThreads")
        Dim i As Int32
        For i = 0 To threadcount - 1
            Dim thread As Thread = Nothing
            thread = New Thread(AddressOf crawl)
            thread.Start()
            crawlthreads.push_back(thread)
        Next

        For i = 0 To crawlthreads.size() - 1
            crawlthreads(i).Join()
        Next
    End Sub

    Public Sub start()
        assert(crawlthreads Is Nothing AndAlso urls Is Nothing AndAlso mutex Is Nothing AndAlso rssreader Is Nothing _
               , "should not call start more than once.")
        assert(config.storoomhosts.size() > 0 AndAlso config.storoomports.size() = config.storoomhosts.size() _
               , "storoomhosts.size() = 0 or storoomhosts.size() <> storoomports.size()")
        urls = New vector(Of String)
        mutex = New Mutex
        rssreader = New rssreader()
        crawlthreads = New vector(Of Thread)
        Try
            Dim storoomIndex As Int32 = 0
            Dim talk As tcpTalk = Nothing

            While True
                crawlthreads.clear()
                copy(config.host, config.storoomhosts(storoomIndex))
                copy(config.port, config.storoomports(storoomIndex))
                raiseError("start to crawl to " + config.host + " : " + Convert.ToString(config.port))
                Dim buff As bytes = Nothing
                If storclient.tryread(talk, config.host, config.port, feedsname, buff, config.storoomretrytimes) _
                                        = returnValue.success Then
                    Dim writecrawlingtag As Boolean = config.writecrawlingtag
                    If writecrawlingtag AndAlso storclient.trymodify(talk, config.host, config.port, crawling, Nothing _
                                                                , config.storoomretrytimes) <> returnValue.success Then
                        raiseError("cannot write crawling tag to storoom.", errorHandle.errorType.warning)
                    End If
                    strsepar(encoding.utf8.GetString(buff.data), spaceChars, urls, True)
                    startCrawlThreads(config.threadcount)
                    If writecrawlingtag AndAlso storclient.trydelete(talk, config.host, config.port, crawling _
                                                                , config.storoomretrytimes) <> returnValue.success Then
                        raiseError("cannot delete crawling tag from storoom.", errorHandle.errorType.warning)
                    End If
                Else
                    raiseError("cannot read feeds list from storoom, ignore this crawling cycle." _
                               , errorHandle.errorType.warning)
                End If
                storoomIndex += 1
                If storoomIndex = config.storoomhosts.size() Then
                    storoomIndex = 0
                End If

                raiseError("sleep for next round of crawling.")
                sleep(config.crawlinterval)
            End While
        Catch
            If Not crawlthreads Is Nothing Then
                Dim i As Int32
                For i = 0 To crawlthreads.size() - 1
                    assert(Not crawlthreads(i) Is Nothing _
                           , "crawlthreads(" + Convert.ToString(i) + ") has been released before service thread ended.")
                    crawlthreads(i).Abort()
                Next
                crawlthreads.clear()
            End If
        End Try
    End Sub

    Public Sub shutdown()
        If Not servicethread Is Nothing Then
            servicethread.Abort()
            servicethread.Join()
        End If
    End Sub

    Public Sub monitor()
        While Not consoleExtSignal()
            sleep()
        End While

        raiseError("caught console cancel key event, shutdown")
        shutdown()
    End Sub

    Public Sub unhandledException(ByVal ex As Exception)
        shutdown()
    End Sub

    Friend unhandledExceptionHandler As UnhandledExceptionDelegate = AddressOf unhandledException

    Public Sub main(ByVal args() As String)
        registerConsoleExtHandle()
        errorHandle.initial(defaultLogFilename())
        registerUnhandledException(unhandledExceptionHandler)
        destructor.start()

        Dim configfile As String = defaultConfigfile
        If args.Length() > 0 Then
            configfile = args(0)
        End If
        If Not config.open(configfile) Then
            raiseError("cannot load config file " + configfile, errorHandle.errorType.exclamation)
        Else
            raiseError("load configuration success, start service")
            servicethread = New Thread(AddressOf start)
            servicethread.Start()
            monitor()
        End If

        destructor.stop()
        errorHandle.finalize()
        consoleExtHandleFinish()
    End Sub

End Module
